|
Jupyter at Bryn Mawr College |
|
|
| Public notebooks: /services/public/dblank / jupyter.cs | |||
Recurrent Neural Networks with Word Embeddings¶
based on:
http://deeplearning.net/tutorial/rnnslu.html
This notebook takes the material from that page, downloads it, and changes it to work with Python3.
First, we delete any left over stuff from a previous run:
In [16]:
!rm -rf is13 atis.pkl*
Now, we get the example code:
In [17]:
!git clone https://github.com/mesnilgr/is13.git
and get the sample data:
In [18]:
!curl -o atis.pkl.gz http://www-etud.iro.umontreal.ca/~mesnilgr/atis/atis.pkl.gz
In [19]:
!gunzip atis.pkl.gz
Now, we convert the example Python2 code to Python3:
In [24]:
!2to3-3.4 -w is13
In [62]:
!sed -i "s/load(f)/load(f, encoding=\"latin1\")/" is13/data/load.py
In [5]:
!sed -i "s/\//\/\//g" is13/utils/tools.py
In [6]:
!sed -i "s/open(filename).read()/bytes(open(filename).read(), \"utf-8\")/" is13/metrics/accuracy.py
In [8]:
!sed -i "s/stdout.split/str(stdout, \"utf-8\").split/" is13/metrics/accuracy.py
Example¶
We import the libraries:
In [1]:
import numpy
import time
import sys
import subprocess
import os
import random
from is13.data import load
from is13.rnn.elman import model
from is13.metrics.accuracy import conlleval
from is13.utils.tools import shuffle, minibatch, contextwin
from functools import reduce
and run an experiment:
In [2]:
s = {'fold':3, # 5 folds 0,1,2,3,4
'lr':0.0627142536696559,
'verbose':1,
'decay':False, # decay on the learning rate if improvement stops
'win':7, # number of words in the context window
'bs':9, # number of backprop through time steps
'nhidden':100, # number of hidden units
'seed':345,
'emb_dimension':100, # dimension of word embedding
'nepochs':50}
folder = "is13/examples"
if not os.path.exists(folder): os.mkdir(folder)
# load the dataset
train_set, valid_set, test_set, dic = load.atisfold(s['fold'])
idx2label = dict((k,v) for v,k in dic['labels2idx'].items())
idx2word = dict((k,v) for v,k in dic['words2idx'].items())
train_lex, train_ne, train_y = train_set
valid_lex, valid_ne, valid_y = valid_set
test_lex, test_ne, test_y = test_set
vocsize = len(set(reduce(\
lambda x, y: list(x)+list(y),\
train_lex+valid_lex+test_lex)))
nclasses = len(set(reduce(\
lambda x, y: list(x)+list(y),\
train_y+test_y+valid_y)))
nsentences = len(train_lex)
# instanciate the model
numpy.random.seed(s['seed'])
random.seed(s['seed'])
rnn = model( nh = s['nhidden'],
nc = nclasses,
ne = vocsize,
de = s['emb_dimension'],
cs = s['win'] )
# train with early stopping on validation set
best_f1 = -numpy.inf
s['clr'] = s['lr']
for e in range(s['nepochs']):
# shuffle
print("Epoch...", e)
sys.stdout.flush()
shuffle([train_lex, train_ne, train_y], s['seed'])
s['ce'] = e
tic = time.time()
for i in range(nsentences):
cwords = contextwin(train_lex[i], s['win'])
words = [numpy.asarray(x).astype('int32') for x in minibatch(cwords, s['bs'])]
labels = train_y[i]
for word_batch , label_last_word in zip(words, labels):
rnn.train(word_batch, label_last_word, s['clr'])
rnn.normalize()
if s['verbose']:
print('[learning] epoch %i >> %2.2f%%'%(e,(i+1)*100./nsentences),'completed in %.2f (sec) <<\r'%(time.time()-tic), end=' ')
sys.stdout.flush()
# evaluation // back into the real world : idx -> words
predictions_test = [ [idx2label[x] for x in rnn.classify(numpy.asarray(contextwin(x, s['win'])).astype('int32'))]\
for x in test_lex ]
groundtruth_test = [ [idx2label[x] for x in y] for y in test_y ]
words_test = [ [idx2word[x] for x in w] for w in test_lex]
predictions_valid = [ [idx2label[x] for x in rnn.classify(numpy.asarray(contextwin(x, s['win'])).astype('int32'))]\
for x in valid_lex ]
groundtruth_valid = [ [idx2label[x] for x in y] for y in valid_y ]
words_valid = [ [idx2word[x] for x in w] for w in valid_lex]
# evaluation // compute the accuracy using conlleval.pl
res_test = conlleval(predictions_test, groundtruth_test, words_test, folder + '/current.test.txt')
res_valid = conlleval(predictions_valid, groundtruth_valid, words_valid, folder + '/current.valid.txt')
if res_valid['f1'] > best_f1:
rnn.save(folder)
best_f1 = res_valid['f1']
if s['verbose']:
print('NEW BEST: epoch', e, 'valid F1', res_valid['f1'], 'best test F1', res_test['f1'], ' '*20)
s['vf1'], s['vp'], s['vr'] = res_valid['f1'], res_valid['p'], res_valid['r']
s['tf1'], s['tp'], s['tr'] = res_test['f1'], res_test['p'], res_test['r']
s['be'] = e
subprocess.call(['mv', folder + '/current.test.txt', folder + '/best.test.txt'])
subprocess.call(['mv', folder + '/current.valid.txt', folder + '/best.valid.txt'])
else:
print('')
# learning rate decay if no improvement in 10 epochs
if s['decay'] and abs(s['be']-s['ce']) >= 10: s['clr'] *= 0.5
if s['clr'] < 1e-5: break
print('BEST RESULT: epoch', e, 'valid F1', s['vf1'], 'best test F1', s['tf1'], 'with the model', folder)